{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline  \n",
    "from matplotlib.font_manager import FontProperties"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Usage\n",
    "\n",
    "This notebook is to visualize the output of `all_in_one_cifar10.py` and `all_in_one_imagenet.py` simply change the `df = pd.read_csv('ifgsm_imagenet_0.03_new.csv')` definition below to read_csv from your specific output file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num batches: 1562\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('ResNet50-bim-new_test.csv')\n",
    "print(\"num batches: \" + str(df[df.with_ILA == True]['batch_index'].max()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualize_result(df):\n",
    "    source_models = df.source_model.unique()\n",
    "    target_models =  df.target_model.unique()\n",
    "    attacks = df.fool_method.unique()\n",
    "    metrics = ['acc_after_attack']\n",
    "    for metric in metrics:\n",
    "\n",
    "        for source_model in source_models:\n",
    "            fig = plt.figure(figsize = [6*len(attacks), 4])\n",
    "            shared_ax = None\n",
    "            for i, attack in enumerate(attacks):\n",
    "\n",
    "                if shared_ax == None:\n",
    "                    shared_ax = plt.subplot(1,len(attacks),i+1)\n",
    "                else:\n",
    "                    plt.subplot(1,len(attacks),i+1, sharey=shared_ax)\n",
    "\n",
    "                layer_indexs = df[df.source_model == source_model]['layer_index'].unique()[1:]\n",
    "                layer_names = df[df.source_model == source_model]['layer_name'].unique()[1:]\n",
    "                print(layer_names)\n",
    "                for target_model in target_models:           \n",
    "                    r = df[(df.source_model == source_model) & (df.target_model == target_model) & (df.fool_method == attack)]\n",
    "\n",
    "                    fla = r[r.with_ILA].groupby('layer_index')\n",
    "                    other = r[r.with_ILA == False]\n",
    "                    xs = fla.layer_index.unique()\n",
    "                    baseline = other[metric].mean()\n",
    "                    fla_r = fla[metric].mean()\n",
    "                    names = fla.layer_name.unique()\n",
    "                    p = plt.plot(layer_indexs,  [baseline for i in xs], linestyle = '--', label = target_model)\n",
    "                    plt.plot(fla_r, label =  target_model + \" ILAP\", color = p[0]._color)\n",
    "                    if source_model != target_model and ((metric == 'fool_rate' and not(fla_r > baseline).any())or (metric == 'acc_after_attack' and not(fla_r < baseline).any())):\n",
    "                        print(\"never perform absolutely better: \" + source_model + \" \" + target_model)\n",
    "\n",
    "                plt.ylim([0,75])\n",
    "                plt.ylabel('Accuracy after Attack')\n",
    "                plt.xlabel('Layer Index')\n",
    "                fontP = FontProperties()\n",
    "                fontP.set_size('small')\n",
    "                \n",
    "                plt.legend(ncol = 3, loc=2, prop=fontP)\n",
    "            plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_model</th>\n",
       "      <th>target_model</th>\n",
       "      <th>original_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>DenseNet121</td>\n",
       "      <td>73.976328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ResNet50</td>\n",
       "      <td>75.691779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG16</td>\n",
       "      <td>71.493122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG19</td>\n",
       "      <td>72.188900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  source_model target_model  original_acc\n",
       "0     ResNet50  DenseNet121     73.976328\n",
       "1     ResNet50     ResNet50     75.691779\n",
       "2     ResNet50        VGG16     71.493122\n",
       "3     ResNet50        VGG19     72.188900"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Original model accuracies\n",
    "# df.groupby(['source_model', 'target_model','fool_method'])['original_acc'].mean().reset_index()\n",
    "df.groupby(['source_model', 'target_model'])['original_acc'].mean().reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['conv1' 'bn1' 'layer1' 'layer2' 'layer3' 'layer4' 'fc']\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'DataFrame' object has no attribute 'fool_method'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-9054265da517>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mvisualize_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-7-a6f91209d8f1>\u001b[0m in \u001b[0;36mvisualize_result\u001b[0;34m(df)\u001b[0m\n\u001b[1;32m     20\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlayer_names\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mtarget_model\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtarget_models\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m                     \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msource_model\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0msource_model\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtarget_model\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mtarget_model\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfool_method\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mattack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m                     \u001b[0mfla\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_ILA\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'layer_index'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/py36torchgpu/lib/python3.6/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   5139\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_info_axis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_can_hold_identifiers_and_holds_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5140\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5141\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5142\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5143\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'fool_method'"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAAD8CAYAAAAoqlyCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAANGElEQVR4nO3cf6jdd33H8efLZp2sqzqWK0gSbWXpaugG7S5dhzA72o20fyR/OCSB4hylQbfKQBl0dHRS/3IyB0I2zVjpFLRW/5ALRgq6SqGYrre01ialco2dTS3rtXb9p9gf7L0/zuk4uyY5X5P3vfec5PmAwPl+z+ee8/7mJM+cc7/3m1QVkqSz96bNHkCSzhUGVZKaGFRJamJQJamJQZWkJgZVkppMDWqSu5I8n+SJU9yfJJ9NspLk8SRX9Y8pSbNvyDvUu4Hdp7n/BmDn+NcB4J/PfixJmj9Tg1pVDwA/O82SvcAXauQI8LYk7+gaUJLmxZaGx9gGPDOxfWK877m1C5McYPQulosuuuj3Lr/88oanl6Q+jzzyyE+rauFMvrYjqINV1SHgEMDi4mItLy9v5NNL0lRJ/vNMv7bjLP+zwI6J7e3jfZJ0XukI6hLwwfHZ/muAl6rqFz7uS9K5bupH/iRfBq4FtiY5Afwd8CsAVfU54DBwI7ACvAz8+XoNK0mzbGpQq2r/lPsL+Mu2iSRpTnmllCQ1MaiS1MSgSlITgypJTQyqJDUxqJLUxKBKUhODKklNDKokNTGoktTEoEpSE4MqSU0MqiQ1MaiS1MSgSlITgypJTQyqJDUxqJLUxKBKUhODKklNDKokNTGoktTEoEpSE4MqSU0MqiQ1MaiS1MSgSlITgypJTQyqJDUxqJLUxKBKUhODKklNDKokNTGoktTEoEpSk0FBTbI7yVNJVpLcdpL735nk/iSPJnk8yY39o0rSbJsa1CQXAAeBG4BdwP4ku9Ys+1vg3qq6EtgH/FP3oJI064a8Q70aWKmq41X1KnAPsHfNmgLeMr79VuAnfSNK0nwYEtRtwDMT2yfG+yZ9ArgpyQngMPDRkz1QkgNJlpMsr66unsG4kjS7uk5K7QfurqrtwI3AF5P8wmNX1aGqWqyqxYWFhaanlqTZMCSozwI7Jra3j/dNuhm4F6Cqvgu8GdjaMaAkzYshQX0Y2Jnk0iQXMjrptLRmzY+B6wCSvIdRUP1ML+m8MjWoVfU6cCtwH/Ako7P5R5PcmWTPeNnHgVuSfA/4MvChqqr1GlqSZtGWIYuq6jCjk02T++6YuH0MeG/vaJI0X7xSSpKaGFRJamJQJamJQZWkJgZVkpoYVElqYlAlqYlBlaQmBlWSmhhUSWpiUCWpiUGVpCYGVZKaGFRJamJQJamJQZWkJgZVkpoYVElqYlAlqYlBlaQmBlWSmhhUSWpiUCWpiUGVpCYGVZKaGFRJamJQJamJQZWkJgZVkpoYVElqYlAlqYlBlaQmBlWSmhhUSWoyKKhJdid5KslKkttOseYDSY4lOZrkS71jStLs2zJtQZILgIPAHwMngIeTLFXVsYk1O4G/Ad5bVS8meft6DSxJs2rIO9SrgZWqOl5VrwL3AHvXrLkFOFhVLwJU1fO9Y0rS7BsS1G3AMxPbJ8b7Jl0GXJbkwSRHkuw+2QMlOZBkOcny6urqmU0sSTOq66TUFmAncC2wH/iXJG9bu6iqDlXVYlUtLiwsND21JM2GIUF9Ftgxsb19vG/SCWCpql6rqh8BP2AUWEk6bwwJ6sPAziSXJrkQ2AcsrVnzdUbvTkmyldG3AI73jSlJs29qUKvqdeBW4D7gSeDeqjqa5M4ke8bL7gNeSHIMuB/466p6Yb2GlqRZlKralCdeXFys5eXlTXluSTqVJI9U1eKZfK1XSklSE4MqSU0MqiQ1MaiS1MSgSlITgypJTQyqJDUxqJLUxKBKUhODKklNDKokNTGoktTEoEpSE4MqSU0MqiQ1MaiS1MSgSlITgypJTQyqJDUxqJLUxKBKUhODKklNDKokNTGoktTEoEpSE4MqSU0MqiQ1MaiS1MSgSlITgypJTQyqJDUxqJLUxKBKUhODKklNDKokNRkU1CS7kzyVZCXJbadZ9/4klWSxb0RJmg9Tg5rkAuAgcAOwC9ifZNdJ1l0M/BXwUPeQkjQPhrxDvRpYqarjVfUqcA+w9yTrPgl8Cvh543ySNDeGBHUb8MzE9onxvv+T5CpgR1V943QPlORAkuUky6urq7/0sJI0y876pFSSNwGfAT4+bW1VHaqqxapaXFhYONunlqSZMiSozwI7Jra3j/e94WLgCuA7SZ4GrgGWPDEl6XwzJKgPAzuTXJrkQmAfsPTGnVX1UlVtrapLquoS4Aiwp6qW12ViSZpRU4NaVa8DtwL3AU8C91bV0SR3Jtmz3gNK0rzYMmRRVR0GDq/Zd8cp1l579mNJ0vzxSilJamJQJamJQZWkJgZVkpoYVElqYlAlqYlBlaQmBlWSmhhUSWpiUCWpiUGVpCYGVZKaGFRJamJQJamJQZWkJgZVkpoYVElqYlAlqYlBlaQmBlWSmhhUSWpiUCWpiUGVpCYGVZKaGFRJamJQJamJQZWkJgZVkpoYVElqYlAlqYlBlaQmBlWSmhhUSWpiUCWpyaCgJtmd5KkkK0luO8n9H0tyLMnjSb6d5F39o0rSbJsa1CQXAAeBG4BdwP4ku9YsexRYrKrfBb4G/H33oJI064a8Q70aWKmq41X1KnAPsHdyQVXdX1UvjzePANt7x5Sk2TckqNuAZya2T4z3ncrNwDdPdkeSA0mWkyyvrq4On1KS5kDrSakkNwGLwKdPdn9VHaqqxapaXFhY6HxqSdp0WwaseRbYMbG9fbzv/0lyPXA78L6qeqVnPEmaH0PeoT4M7ExyaZILgX3A0uSCJFcCnwf2VNXz/WNK0uybGtSqeh24FbgPeBK4t6qOJrkzyZ7xsk8Dvw58NcljSZZO8XCSdM4a8pGfqjoMHF6z746J29c3zyVJc8crpSSpiUGVpCYGVZKaGFRJamJQJamJQZWkJgZVkpoYVElqYlAlqYlBlaQmBlWSmhhUSWpiUCWpiUGVpCYGVZKaGFRJamJQJamJQZWkJgZVkpoYVElqYlAlqYlBlaQmBlWSmhhUSWpiUCWpiUGVpCYGVZKaGFRJamJQJamJQZWkJgZVkpoYVElqYlAlqYlBlaQmBlWSmgwKapLdSZ5KspLktpPc/6tJvjK+/6Ekl7RPKkkzbmpQk1wAHARuAHYB+5PsWrPsZuDFqvot4B+BT3UPKkmzbsg71KuBlao6XlWvAvcAe9es2Qv82/j214DrkqRvTEmafVsGrNkGPDOxfQL4/VOtqarXk7wE/Cbw08lFSQ4AB8abryR54kyGnkNbWfN7cQ7zWM9N59Ox/vaZfuGQoLapqkPAIYAky1W1uJHPv1k81nOTx3puSrJ8pl875CP/s8COie3t430nXZNkC/BW4IUzHUqS5tGQoD4M7ExyaZILgX3A0po1S8CfjW//KfDvVVV9Y0rS7Jv6kX/8PdFbgfuAC4C7qupokjuB5apaAv4V+GKSFeBnjKI7zaGzmHveeKznJo/13HTGxxrfSEpSD6+UkqQmBlWSmqx7UM+ny1YHHOvHkhxL8niSbyd512bM2WHasU6se3+SSjK3P3Iz5FiTfGD82h5N8qWNnrHLgD/D70xyf5JHx3+Ob9yMOTskuSvJ86f6efiMfHb8e/F4kqumPmhVrdsvRiexfgi8G7gQ+B6wa82avwA+N769D/jKes60ycf6R8CvjW9/5Fw+1vG6i4EHgCPA4mbPvY6v607gUeA3xttv3+y51/FYDwEfGd/eBTy92XOfxfH+IXAV8MQp7r8R+CYQ4BrgoWmPud7vUM+ny1anHmtV3V9VL483jzD6md55NOR1Bfgko//X4ecbOVyzIcd6C3Cwql4EqKrnN3jGLkOOtYC3jG+/FfjJBs7XqqoeYPRTSaeyF/hCjRwB3pbkHad7zPUO6skuW912qjVV9TrwxmWr82bIsU66mdG/fvNo6rGOPx7tqKpvbORg62DI63oZcFmSB5McSbJ7w6brNeRYPwHclOQEcBj46MaMtil+2b/TG3vpqUaS3AQsAu/b7FnWQ5I3AZ8BPrTJo2yULYw+9l/L6FPHA0l+p6r+ezOHWif7gbur6h+S/AGjnz+/oqr+Z7MHmwXr/Q71fLpsdcixkuR64HZgT1W9skGzdZt2rBcDVwDfSfI0o+8/Lc3piakhr+sJYKmqXquqHwE/YBTYeTPkWG8G7gWoqu8Cb2b0H6eciwb9nZ603kE9ny5bnXqsSa4EPs8opvP6fTaYcqxV9VJVba2qS6rqEkbfL95TVWf8n05soiF/hr/O6N0pSbYy+hbA8Q2cscuQY/0xcB1AkvcwCurqhk65cZaAD47P9l8DvFRVz532KzbgTNqNjP7F/iFw+3jfnYz+gsHoBfkqsAL8B/DuzT77t47H+i3gv4DHxr+WNnvm9TrWNWu/w5ye5R/4uobRtziOAd8H9m32zOt4rLuABxn9BMBjwJ9s9sxncaxfBp4DXmP0KeNm4MPAhyde14Pj34vvD/kz7KWnktTEK6UkqYlBlaQmBlWSmhhUSWpiUCWpiUGVpCYGVZKa/C+6Stl1hjbYSQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1296x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "visualize_result(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_model</th>\n",
       "      <th>target_model</th>\n",
       "      <th>fool_method</th>\n",
       "      <th>acc_after_attack</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>DenseNet121</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>45.771353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>0.031990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG16</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>50.203935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG19</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>52.177303</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  source_model target_model fool_method  acc_after_attack\n",
       "0     ResNet50  DenseNet121       ifgsm         45.771353\n",
       "1     ResNet50     ResNet50       ifgsm          0.031990\n",
       "2     ResNet50        VGG16       ifgsm         50.203935\n",
       "3     ResNet50        VGG19       ifgsm         52.177303"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseline = df[df.with_ILA == False].groupby(['source_model', 'target_model','fool_method'])['acc_after_attack'].mean().reset_index()\n",
    "baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_model</th>\n",
       "      <th>target_model</th>\n",
       "      <th>fool_method</th>\n",
       "      <th>layer_index</th>\n",
       "      <th>acc_after_attack</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>DenseNet121</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>0.0</td>\n",
       "      <td>33.305342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>DenseNet121</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>1.0</td>\n",
       "      <td>45.637396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>DenseNet121</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>2.0</td>\n",
       "      <td>37.072137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>DenseNet121</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>3.0</td>\n",
       "      <td>29.228647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>DenseNet121</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>4.0</td>\n",
       "      <td>32.431622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>DenseNet121</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>5.0</td>\n",
       "      <td>53.438900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>DenseNet121</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>6.0</td>\n",
       "      <td>55.768154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.135956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.199936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.371881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.139955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.115963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.131958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ResNet50</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.133957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG16</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>0.0</td>\n",
       "      <td>38.525672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG16</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>1.0</td>\n",
       "      <td>45.603407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG16</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>2.0</td>\n",
       "      <td>38.687620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG16</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>3.0</td>\n",
       "      <td>34.231046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG16</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>4.0</td>\n",
       "      <td>39.267434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG16</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>5.0</td>\n",
       "      <td>55.588212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG16</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>6.0</td>\n",
       "      <td>57.011756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG19</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>0.0</td>\n",
       "      <td>39.863244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG19</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>1.0</td>\n",
       "      <td>47.396833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG19</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>2.0</td>\n",
       "      <td>40.365083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG19</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>3.0</td>\n",
       "      <td>35.762556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG19</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40.698976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG19</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>5.0</td>\n",
       "      <td>57.087732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>ResNet50</td>\n",
       "      <td>VGG19</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>6.0</td>\n",
       "      <td>58.335333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_model target_model fool_method  layer_index  acc_after_attack\n",
       "0      ResNet50  DenseNet121       ifgsm          0.0         33.305342\n",
       "1      ResNet50  DenseNet121       ifgsm          1.0         45.637396\n",
       "2      ResNet50  DenseNet121       ifgsm          2.0         37.072137\n",
       "3      ResNet50  DenseNet121       ifgsm          3.0         29.228647\n",
       "4      ResNet50  DenseNet121       ifgsm          4.0         32.431622\n",
       "5      ResNet50  DenseNet121       ifgsm          5.0         53.438900\n",
       "6      ResNet50  DenseNet121       ifgsm          6.0         55.768154\n",
       "7      ResNet50     ResNet50       ifgsm          0.0          0.135956\n",
       "8      ResNet50     ResNet50       ifgsm          1.0          0.199936\n",
       "9      ResNet50     ResNet50       ifgsm          2.0          0.371881\n",
       "10     ResNet50     ResNet50       ifgsm          3.0          0.139955\n",
       "11     ResNet50     ResNet50       ifgsm          4.0          0.115963\n",
       "12     ResNet50     ResNet50       ifgsm          5.0          0.131958\n",
       "13     ResNet50     ResNet50       ifgsm          6.0          0.133957\n",
       "14     ResNet50        VGG16       ifgsm          0.0         38.525672\n",
       "15     ResNet50        VGG16       ifgsm          1.0         45.603407\n",
       "16     ResNet50        VGG16       ifgsm          2.0         38.687620\n",
       "17     ResNet50        VGG16       ifgsm          3.0         34.231046\n",
       "18     ResNet50        VGG16       ifgsm          4.0         39.267434\n",
       "19     ResNet50        VGG16       ifgsm          5.0         55.588212\n",
       "20     ResNet50        VGG16       ifgsm          6.0         57.011756\n",
       "21     ResNet50        VGG19       ifgsm          0.0         39.863244\n",
       "22     ResNet50        VGG19       ifgsm          1.0         47.396833\n",
       "23     ResNet50        VGG19       ifgsm          2.0         40.365083\n",
       "24     ResNet50        VGG19       ifgsm          3.0         35.762556\n",
       "25     ResNet50        VGG19       ifgsm          4.0         40.698976\n",
       "26     ResNet50        VGG19       ifgsm          5.0         57.087732\n",
       "27     ResNet50        VGG19       ifgsm          6.0         58.335333"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ILA\n",
    "q = df[(df.with_ILA == True)].groupby(['source_model', 'target_model','fool_method','layer_index'])['acc_after_attack'].mean().reset_index()\n",
    "baseline.head()\n",
    "q\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
